#include <math.h>
#include <setjmp.h>
#include <stdio.h>
#include <strings.h>

#include "CaffCache/hash_table.h"
#include "CaffProc/thread.h"
#include "CaffProc/processor.h"
#include "CaffProc/processor_context.h"
#include "CaffMC/memory.h"
#include "uarch/module.h"
#include "uarch/mreq.h"
#include "uarch/settings.h"
#include "uarch/signals.h"
#include "uarch/sim.h"
#include "uarch/types.h"
#include "uarch/ticker.h"
//#include "../config/config_params.h"
//#include "CaffNet/noc.h"


using namespace manifold::kernel;

/** Global Simulator object.  */
Simulator Sim;
Sim_settings *Settings = &(Sim.settings);
///////////////////////////////////////////////////////////////////////////////////////
extern void core_tick_handler();
///////////////////////////////////////////////////////////////////////////////////////
//static bool warming;
//static long long int total_insn;

bool sim_fini = false;
bool end_benchmark = false;

/** Fatal Error.  */
void fatal_error (const char *fmt, ...)
{
    va_list ap;

    va_start (ap, fmt);
    vfprintf (stderr, fmt, ap);
    va_end (ap);
    
    /** Dump stats, etc. before we fatal error.  */
    Sim.fini ();

    /** Enable debugging by asserting zero.  */
    assert (0 && "Fatal Error");
    exit (-1);
}

Simulator::Simulator ()
{
    /** Init signal handlers.  */
    signal_hander_init ();

    /** Get the initial settings.  */
    settings.get_settings ();
    settings.get_topology ();

    /** Basic assumptions about setup.  */
    if (settings.num_nodes == 0)
        fatal_error ("Number of nodes is ZERO!");

    /** Check to make sure debug is disabled if this is a regression run.  */
    if (settings.regression_test && 
        (GENERAL_DEBUG || TICK_TOCK_DEBUG || Sim.settings.debug_addr != 0x0))
        fatal_error ("DEBUG enabled during regression.\n");

    if (settings.regression_test && Sim.settings.test_addr != 0x0)
        fatal_error ("test_addr non-zero during regression.\n");

    if (settings.qsets_enabled && settings.sel_rep_pred != INVALID_PRED)
        fatal_error ("remap table active with Qsets turned on?\n");

    if (settings.cache_index_swizzle & settings.dir_home_swizzle)
        fatal_error ("Cache index and home node swizzle overlap!\n");

    /** Set global_clock to cycle zero.  */
    global_clock = 0;

    /** Set number of simulated instructions.  */
    //settings.sesc_nsim = settings.sesc_nsim_per_core * settings.num_nodes * settings.threads_per_processor;
    //settings.warmup_time = settings.warmup_time_per_core * settings.num_nodes * settings.threads_per_processor;

    if (settings.sesc_nsim)
        fprintf (stderr, "Simulation based on instruction count should be depreciated.");

    /** Stats.  */
    stats = new Simulator_stat_engine();
    stat_manager = new Sim_stat_manager(settings.report_output);
    stat_manager->add_simulator_stat_engine(stats);

    /** Seed random number generator.  */
    srandom (1023);

#if 0    // This is where Manifold takes over instantiation

    NOC = new Noc ();
/*
    NOC->build_flat_bus (settings.num_nodes, LINK_FLIT_WIDTH);
*/

/*
    NOC->build_flat_mesh (settings.mesh_x_dimension,
                         settings.mesh_y_dimension,
                         LINK_FLIT_WIDTH,
                         settings.num_virtual_channels,
                         settings.buffer_entries_per_vc);
*/

    //ugly hard corded values for testing @ 64!

/*
    NOC->build_bus_bus  (8, LINK_FLIT_WIDTH,
                         8, LINK_FLIT_WIDTH);
*/
/*
    NOC->build_mesh_mesh (4, 4,
                         LINK_FLIT_WIDTH,
                         settings.num_virtual_channels,
                         settings.buffer_entries_per_vc,

                         2, 4,
                         LINK_FLIT_WIDTH,
                         settings.num_virtual_channels,
                         settings.buffer_entries_per_vc);
*/
/*
    NOC->build_mesh_bus (4, 4,
                         LINK_FLIT_WIDTH,
                         settings.num_virtual_channels,
                         settings.buffer_entries_per_vc,

                         4,
                         LINK_FLIT_WIDTH);
*/
/*
    NOC->build_mesh_bus_bus (4, 4,
                         LINK_FLIT_WIDTH,
                         settings.num_virtual_channels,
                         settings.buffer_entries_per_vc,

                         2, LINK_FLIT_WIDTH,
                         4, LINK_FLIT_WIDTH);
*/

    if(settings.noc_topology == FLAT_MESH)
    {
        //assert(settings.num_nodes == 128);
        NOC->build_flat_mesh (settings.mesh_x_dimension,
                             settings.mesh_y_dimension,
                             LINK_FLIT_WIDTH,
                             settings.num_virtual_channels,
                             settings.buffer_entries_per_vc);

    }
    else if(settings.noc_topology == PIRANHA)
    {
        assert(settings.num_nodes == 128);
        NOC->build_mesh_mesh (4, 4,
                             LINK_FLIT_WIDTH,
                             settings.num_virtual_channels,
                             settings.buffer_entries_per_vc,

                             2, 4,
                             LINK_FLIT_WIDTH,
                             settings.num_virtual_channels,
                             settings.buffer_entries_per_vc);
    }
    else if(settings.noc_topology == SUPERDOME)
    {
        assert(settings.num_nodes == 128);
        NOC->build_mesh_bus_bus (4, 4,
                             LINK_FLIT_WIDTH,
                             settings.num_virtual_channels,
                             settings.buffer_entries_per_vc,

                             2, LINK_FLIT_WIDTH,
                             4, LINK_FLIT_WIDTH);

    }

    Nd = new Node*[settings.num_nodes];

    /** Allocate cores, caches, directories, and memory controllers.  */
    for (int node = 0; node < settings.num_nodes; node++)
    {
//        Mesh_interface *NI;
        Network_interface *NI;

//        NI = Sim.NOC->get_mesh_interface (node);
        NI = Sim.NOC->get_interface_module_for_id (node);
        assert (NI != NULL && "something in noc construction screwed up!");
        Nd[node] = new Node (node);

        /** Core. */
/*
        if (settings.dir_mode == DIR_1L)
            //Nd[node]->build_1Level_MESI_node (NI);
            Nd[node]->build_1Level_MOESI_node (NI);
        else if (settings.dir_mode == DIR_2L_FULL_REP)
            //Nd[node]->build_2Level_FULL_REP_MESI_node (NI);
            Nd[node]->build_2Level_FULL_REP_MOESI_node (NI);
        else if (settings.dir_mode == DIR_2L_SELECT_REP)
            Nd[node]->build_2Level_SELECT_REP_MESI_node (NI);
*/

        if (settings.dir_tiers == 1)
            Nd[node]->build_1Level_MOESI_node (NI);
        else if (settings.noc_topology == FLAT_MESH)
                Nd[node]->build_2Level_FULL_REP_MOESI_node (NI);
        else if (settings.noc_topology == PIRANHA)
                Nd[node]->build_2Level_FULL_REP_MOESI_node (NI);
        else if (settings.noc_topology == SUPERDOME)
                Nd[node]->build_2Level_SUPERDOME_MOESI_node (NI);
		
    }

    /** Allocate memory controllers.  */
    for (int mc = 0; mc < settings.num_mem_ctrls; mc++)
    {
        Nd[settings.mem_ctrl_array[mc]]->add_MC_to_node ();
    }
    
    /** Setup neighborhoods.  */
    if (settings.num_nhoods > 0)
    {
        int xbf = settings.nhood_x_blocking_factor;
        int ybf = settings.nhood_y_blocking_factor;

        if (settings.num_nhoods != (settings.num_nodes / (xbf * ybf)))
            fatal_error ("Settings nhoods not equivalent to calculated nhoods.\n");

        if (settings.num_nodes % (xbf * ybf) != 0)
            fatal_error ("Neighborhood blocking factor not valid!\n");
    }

#endif //end of instantiation

#if 0 //new repeaters will use manifold
    Repeater_event *event;

    event = (Repeater_event *) new TRepeater_event<Simulator> (this, &Simulator::thread_count, true, 4096);
    add_repeater_event(event);

    event = (Repeater_event *) new TRepeater_event<Simulator> (this, &Simulator::livelock_check, true, settings.heartrate);
    add_repeater_event(event);

    if(Sim.settings.qsets_enabled)
    {
        event = (Repeater_event *) new TRepeater_event<Simulator> (this, &Simulator::collect_qset_stat, true, settings.qsets_interval);
        add_repeater_event(event);
    }
#endif //end repeaters

}

void Simulator::Manifold_instantiate ()
{

    /* 
    NOC = new Noc ();
    NOC->build_fakenet (settings.num_nodes);
     * */

    
    Nd = new Node*[settings.num_nodes];

    assert ( settings.num_nodes == tp->no_nodes && "number of nodes do not match between Caff and Iris"); 
    /** Allocate cores, caches, directories, and memory controllers.  */
    for (int node = 0; node < settings.num_nodes; node++)
    {
//        CompId_t NI_cid;

//        NI_cid = Sim.NOC->get_interface_cid_for_id (node);
//        assert (NI_cid != -1 && "something in noc construction screwed up! (or is null a valid Cid?)");
        Nd[node] = new Node (node);

        // Creating iris_node_module components for replacing the
        // fakenet_interface to the NOC
        // This is done here because the NOC object and noc_build is no longer
        // used and may be replaced by the topology pointer from IRIS
        CompId_t iris_node_module_cid = Component::Create<Node_iris_module, ModuleID> (0, (ModuleID){node, NI_M});
        Node_iris_module* iris_node_module_ptr = (Node_iris_module*) Component::GetComponent(iris_node_module_cid);
        iris_node_module_ptr->node_terminal = (NodeTerminal*)tp->terminals[node];
        static_cast<NodeTerminal*>(tp->terminals[node])->caffTerm = iris_node_module_ptr;
        Clock::Register<Node_iris_module> (iris_node_module_ptr, &Node_iris_module::tick, &Node_iris_module::tock);

        Nd[node]->build_manifold_node(iris_node_module_cid);

    }

    /** Allocate memory controllers.  */
    for (int mc = 0; mc < settings.num_mem_ctrls; mc++)
    {
        Nd[settings.mem_ctrl_array[mc]]->add_MC_to_manifold_node ();
    }

    /** Setup neighborhoods.  */
    if (settings.num_nhoods > 0)
    {
        int xbf = settings.nhood_x_blocking_factor;
        int ybf = settings.nhood_y_blocking_factor;

        if (settings.num_nhoods != (settings.num_nodes / (xbf * ybf)))
            fatal_error ("Settings nhoods not equivalent to calculated nhoods.\n");

        if (settings.num_nodes % (xbf * ybf) != 0)
            fatal_error ("Neighborhood blocking factor not valid!\n");
    }


    /** Repeaters using Manifold */
    //TODO!!!

    Repeater_event *event;

    event = (Repeater_event *) new TRepeater_event<Simulator> (this, &Simulator::thread_count, true, 4096);
    add_repeater_event(event);

    event = (Repeater_event *) new TRepeater_event<Simulator> (this, &Simulator::livelock_check, true, settings.heartrate);
    add_repeater_event(event);

}

Simulator::~Simulator ()
{
    for (int i = 0; i < settings.num_nodes; i++)
    {
        delete Nd[i];
    }

//    delete NOC;
    delete [] Nd;

    delete stat_manager;
    delete stats;

    std::multimap<timestamp_t, Repeater_event *>::iterator it;
    for (it = repeater_event_queue.begin (); it != repeater_event_queue.end (); it++)
        delete (*it).second;
}

void Simulator::dump_stats ()
{
    stats->cycle_count = Global_Clock;

    //TODO implement a parameter to selectively output all stats with a system summary,
    //     or just the summary summary
    stat_manager->output_all ();
}

/** Done simulating.  */
void Simulator::fini ()
{
    if (sim_fini)
    {
        fprintf (stderr, "Final statistics already dumped, ignoring fini call.\n");
        return;
    }
    sim_fini = true;

    fprintf (stderr, "Caffeine Sim - Ends: %lld\n\n\n", settings.sesc_nsim);
    dump_stats ();
}

void Simulator::livelock_check (void)
{
#if !SIM_WITH_ZESTO
    int active;
    bool livelock_detected;
    Preq *oldest_req;
    LIST<Processor_context *>::iterator context_it;
    LIST<Preq *>::iterator preq_it;

    fprintf(stderr, ".(%lld)", settings.sesc_nsim);

    if (settings.livelock_check)
    {
        oldest_req = NULL;
        active = 0;
        livelock_detected = false;

        for (int i = 0; i < settings.num_nodes; i++)
        {
            //For each core.  
            active += get_PR (i)->active_contexts ();

            for (context_it = get_PR(i)->active_context_list.begin();
                 context_it != get_PR(i)->active_context_list.end();
                 context_it++)
            {
                if ((*context_it)-> outstanding_req_list.empty ())
                    continue;

                for (preq_it = (*context_it)->outstanding_req_list.begin();
                     preq_it != (*context_it)->outstanding_req_list.end();
                     preq_it++)
                {
                    // If the diff between clock and req_time greater then HEARTRATE we have livelock.  
                    if (oldest_req == NULL)
                    {
                        if ((Global_Clock - (*preq_it)->req_time) > settings.heartrate)
                        {
                            oldest_req = (*preq_it);
                            livelock_detected = true;
                        }
                    }
                    else
                    {
                        if (oldest_req->req_time > (*preq_it)->req_time)
                        {
                            oldest_req = (*preq_it);
                        }
                    }
                }
            }
        }

        //if (active <= 5) 
        //    fprintf(stderr, "\nLess than 5 threads active!\n");

        if (livelock_detected)
        {
            paddr_t addr;
            int home;

            addr = oldest_req->addr;
            fprintf (stderr, "\n\nLivelock detected:\nCore: %d\nCycle: %llu\n",
                     oldest_req->mid.nodeID, (unsigned long long int)Global_Clock);
            Sim.dump_outstanding_requests (oldest_req->mid.nodeID);

            /** Additional debug information on the oldest outstanding.  */
            fprintf (stderr, "L1 entry:\n");
            get_L1 (oldest_req->mid.nodeID)->dump_hash_entry (addr);

            fprintf (stderr, "\nL2 entry:\n");
            home = get_L1(oldest_req->mid.nodeID)->get_set(addr)->get_default_dir(addr).nodeID;
            get_L2 (home)->dump_hash_entry (addr);

            if (settings.dir_mode == DIR_2L_FULL_REP || settings.dir_mode == DIR_2L_SELECT_REP)
            {
                fprintf (stderr, "\nL3 entry:\n");
                home = get_L2(home)->get_set(addr)->get_default_dir(addr).nodeID;
                get_L3 (home)->dump_hash_entry (addr);
            }

            if (Sim.settings.regression_test) {
                fprintf (stderr, "MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\n");
                fprintf (stderr, "MMMMMI  MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM...MMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\n");
                fprintf (stderr, "MMMM.     MMM     MM     M     M     .MMMMM     MM.   M          M.    .MMMMM\n");
                fprintf (stderr, "MMM=      MMM     MM    .M     M     .MMMM       M    M   ?      M.      MMMM\n");
                fprintf (stderr, "MMM.       MM     MM    .M     M     .MMMM       M.   M   M      M.      MMMM\n");
                fprintf (stderr, "MMM    .   MM     MM     M     M   ...MMMM   M.  M    M   M     .M.  M.  MMMM\n");
                fprintf (stderr, "MMM    .   MM  .  .M     M     M   .MMMMMM   M.  M    M   M    MMM.  M   MMMM\n");
                fprintf (stderr, "MMM.       MI  I   M     M     M   .MMMMMM   M.  M   .I  .M    MMM.  M.  MMMM\n");
                fprintf (stderr, "MMM.      .M   M   M     M $   M   .MMMMMM   M.  M$   7  .M    MMM.  M   MMMM\n");
                fprintf (stderr, "MMM.   MMMMM   M.  M   7 7 M   M      MMMM   M.  MM       M      M.     .MMMM\n");
                fprintf (stderr, "MMM.       M   M   M   7   M   M      MMMM   M.  MM       M      M.    .MMMMM\n");
                fprintf (stderr, "MMM.       M   M.  M   I   M   M      MMMM   M.  MM       M      M.      MMMM\n");
                fprintf (stderr, "MMM.       M   M   M   I   M   M    7IMMMM   M.  MM.     .M    I7M.  M.  MMMM\n");
                fprintf (stderr, "MMM.  ..   M       M   7  .M   M   .MMMMMM   M.  MM      IM    MMM.  M   MMMM\n");
                fprintf (stderr, "MMM.   .   M       M   7   M   M   .MMMMMM   M.  MM      MM    MMM.  M   MMMM\n");
                fprintf (stderr, "MMM.   .   M       M   I   M   M   .MMMMMM   M.  MM      MM    MMM.  M   MMMM\n");
                fprintf (stderr, "MMM.  ..   M   .   I   7  .M   M      MMMM   M   MM.     MM      M.  M   MMMM\n");
                fprintf (stderr, "MMM        $   M       7.  M   M      MMMM       MM     .MM      M.  M   MMMM\n");
                fprintf (stderr, "MMMM    .      M   .   7  .M   M      MMMM       MMI     MM      M.  M   MMMM\n");
                fprintf (stderr, "MMMM.  .I      M   .   7. .M   M      MMMMM     MMMM    .MM      M.  M   MMMM\n");
                fprintf (stderr, "MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM\n");
            }
            fatal_error ("\nSimulation error: Livelock detected\n");
        }
    }
#endif
}

void Simulator::thread_count ()
{
    int active = 0;
    for (int i = 0; i < settings.num_nodes; i++)
    {
        //active += get_PR (i)->active_contexts ();
    }
    Sim.stats->active_threads_hist.collect (active);
}

/** Home node id given address.  */
int Simulator::get_home_node_id (paddr_t addr)
{
    //this effectively makes the home mapping use a somewhat modulo mapping
    //addr = (addr / 64) * 63;
    addr = (addr >> 6) * 63;

    paddr_t home;

    if (!settings.dir_home_swizzle)
    {
        home = (addr >> settings.dir_addr_per_node_log2) % settings.num_nodes;
    }
    else
    {
        int curr_bit;
        int home_bit;
        paddr_t mask;

        /** TODO: Assumes 32-bit emulation.  */
        home = 0;
        home_bit = 0;

        for (curr_bit = settings.cache_line_size_log2; curr_bit < 32; curr_bit++)
        {
            mask = 1 << curr_bit;
            if (settings.dir_home_swizzle & mask)
            {
                home |= ((addr & mask) >> curr_bit) << home_bit++;
            }
        }
        home = home % settings.num_nodes;
    }
    return home;
}

int Simulator::get_local_replica_node_id (int origin_node_id, paddr_t addr)
{
    assert (Sim.settings.dir_tiers > 1 && "why is this called in single tier?");
    if(Sim.settings.noc_topology != FLAT_MESH)
    {
        //nhood_blocking not really used, but quick hack to get nhood size
        int nodes_per_nhood = settings.nhood_x_blocking_factor * settings.nhood_y_blocking_factor;

        int global_home;
        int global_home_nhood_offset;
        int origin_node_nhood_offset;
        int dest;

        global_home = get_home_node_id (addr);
        global_home_nhood_offset = global_home % nodes_per_nhood;
        origin_node_nhood_offset = origin_node_id % nodes_per_nhood;

        dest = origin_node_id - origin_node_nhood_offset + global_home_nhood_offset;

        return dest;
    }
    else
    {
        //TODO:  Work in progress
        int global_home_id;
        int global_home_x, global_home_y;
        int origin_x, origin_y;
        int dest_x, dest_y;
        int dest_id;

        int gx_offset, gy_offset, ox_offset, oy_offset;

        /** Calculate */
        /** Determine home and origin x,y. */
        /** Use x,y to determine x,y offset within their respective neighborhoods. */
        /** Compute difference in offsets and add to origin x,y. */
        /** Convert back into nodeID. */



        global_home_id = get_home_node_id (addr);
        global_home_x = global_home_id % settings.mesh_x_dimension;
        global_home_y = global_home_id / settings.mesh_x_dimension;

        origin_x = origin_node_id % settings.mesh_x_dimension;
        origin_y = origin_node_id / settings.mesh_x_dimension;

        gx_offset = global_home_x % settings.nhood_x_blocking_factor;
        gy_offset = global_home_y % settings.nhood_y_blocking_factor;
        ox_offset = origin_x % settings.nhood_x_blocking_factor;
        oy_offset = origin_y % settings.nhood_y_blocking_factor;

        dest_x = origin_x + gx_offset - ox_offset;
        dest_y = origin_y + gy_offset - oy_offset;

        dest_id = dest_y * settings.mesh_x_dimension + dest_x;

        return dest_id;

    }
}

//TODO: Memory controller mapping should be using address!
/************ FIX THIS!!!! */
int Simulator::get_memory_controller_id (paddr_t addr)
{
    int mem_controller_hash;

    mem_controller_hash = (addr >> settings.dir_addr_per_node_log2) % settings.num_mem_ctrls;
    return settings.mem_ctrl_array[mem_controller_hash];
}

module_t Simulator::get_querier_module_index (module_t module_index)
{
    //TODO: Clean this up
    return (module_t)((int)module_index - 1);
}

module_t Simulator::get_directory_module_index (module_t module_index)
{
    //TODO: Clean this up
    return (module_t)((int)module_index + 1);
}

/**
 * Run the simulator.
 */
void Simulator::run ()
{
    bool work;
    int heart;
    counter_t hr;

    /** This must match what's in enums.h.  */
    /*
    //unused variables?  why were these removed?
    const char *cp_str[5] = {"UNKNOWN","MSI","MESI","MOSI"};
    const char *dr_str[5] = {"UNKNOWN","DIR_1L","DIR_2L_FULL_REP","DIR_2L_SELECT_REP"};
    const char *dr_pr_str[2] = {"-","THRESHOLD_PRED"};
    const char *dr_pr_scope_str[4] = {"-","NODE","NHOOD","GLOBAL"};
    const char *nt_str[4] = {"UNKNOWN","MESH","EXPRESS_MESH","CN_TORUS"};
     */
    const char *noc_str[4] = {"UNKNOWN","FLAT_MESH","PIRANHA","SUPERDOME"};

    //	total_insn = settings.sesc_nsim;
    //	warming = true;

    /** heart = log2 (heartrate);  */
    hr = settings.heartrate;
    for (heart = 0; hr > 0; hr = hr >> 1, heart++);

    fprintf (stderr, "Caffeine Sim - Begins  ");
    fprintf (stderr, " Heartrate: %d", heart - 1);
    fprintf (stderr, " Num Sim: %lld", settings.sesc_nsim);
    fprintf (stderr, " Cores: %d", settings.num_nodes);
    fprintf (stderr, " MSHRs: %d", settings.mshrs_per_processor); 
    //    fprintf (stderr, " Cache: %s/%s", "LLS", cp_str[settings.dir_coherence_policy[0]]);
    //    fprintf (stderr, " Dir: %s/%s/%s", dr_str[settings.dir_mode],
    //             dr_pr_str[settings.sel_rep_pred], dr_pr_scope_str[settings.sel_rep_pred_scope]);
    //    fprintf (stderr, " Mesh Topology: %s", nt_str[settings.mesh_topology]);
    fprintf (stderr, " Noc Topology: %s", noc_str[settings.noc_topology]);
    fprintf (stderr, " Inf Mesh BW: %c", settings.net_infinite_bw == true ? 't' : 'f');
    fprintf (stderr, " Debug Addr: 0x%8llx\n", (unsigned long long int)settings.debug_addr);

    /** Main run loop.  */
    work = true;



    //        Manifold::StopAtTime(0.2);
    CompId_t ticker_id;
    Ticker *ticker;

    ticker_id = Component::Create<Ticker>(0);
    ticker = (Ticker *) Component::GetComponent(ticker_id); 
    Clock::Register<Ticker> ((Ticker *)ticker, &Ticker::tick, &Ticker::tock);

    Manifold::Run();
    cout << "Simulation complete" << endl;
    //        MPI_Finalize();

#if 0
    while (work && !end_benchmark) 
    {
        if (warming && settings.warmup_time && (total_insn - settings.sesc_nsim) > settings.warmup_time) 
        {
            fprintf(stderr,"Warmup ends: %lld", settings.sesc_nsim);
            stat_manager->clear_all();
            warming = false;
        }

        work = tick_tock ();

        /** Repeater event processing. */
        process_repeater_event_queue ();

        /** Only sample these stats on sampling cycles to prevent major slowdowns. */
        if (!warming &&
            Sim.settings.qsets_enabled &&
            (Global_Clock % Sim.settings.qsets_interval) == 0)
        {
            int followers_1lvl = 0;
            for (int i = 0; i < settings.num_nodes; i++)
            {
                /** Assumes set zero is a follower set!!! */
                if (get_L1(i)->get_set(0)->dir_hash == HASH_NODE_STRIDE)
                    followers_1lvl++;
            }
            Sim.stats->qset_1lvl_hist.collect (followers_1lvl);
        }

        /** Finished simulating. TODO: Really shouldn't be simulating based on instruction count.  */
        //        if (settings.sesc_nsim < 0)
        //            break;

    }
#endif

}

/** Simulate the advancement of the system clock.  */
bool Simulator::tick_tock ()
{

    bool work = true;
    static int sched_processor = 0;

    /*Ticks all the zesto cores*/
#if SIM_WITH_ZESTO

    // for (int i = 0; i < settings.num_nodes; i++)
    // {
    //   Nd[i]->mod[PR_M]->tick();
    // }

    //core_tick_handler();
#else
#if 0
    work = false;
    for (int i = sched_processor; i < settings.num_nodes; i++)
    {
        if (get_PR(i)->active_contexts () > 0 && get_PR(i)->tick_tock_p ())
        {
            get_PR(i)->tick ();
            work = true;
        }
    }


    for (int i = 0; i < sched_processor; i++)
    {
        if (get_PR(i)->active_contexts () > 0 && get_PR(i)->tick_tock_p ())
        {
            get_PR(i)->tick ();
            work = true;
        }
    }

    (sched_processor < settings.num_nodes) ? sched_processor++ : sched_processor = 0;
#endif
#endif

    /** Advance global clock.  */
    global_clock++;

    /** Ticks.  */
    for (int i = 0; i < settings.num_nodes; i++)
        Nd[i]->tick ();

//    NOC->tick ();
//    NOC->tock ();

    /** Tocks.  */
    for (int i = 0; i < settings.num_nodes; i++)
        Nd[i]->tock ();

    return work;

}

/** Flush all requests from simulator.  */
void Simulator::flush ()
{
    bool work = true;
    LIST<Processor_context *>::iterator context_it;
    processor *pr;

    fprintf (stderr, "Caffeine Sim - Flushing outstanding requests..\n");

    /*    while (work)
          {
          work = false;
          for (int i = 0; i < settings.num_nodes; i++)
          {
          pr = get_PR (i);
          for (context_it = pr->active_context_list.begin(); context_it != pr->active_context_list.end(); context_it++)
          {
          if ((*context_it)->outstanding_reqs)
          {
          work = true;
          break;
          }
          }
          }
          if (work)
          tick_tock ();
          }
     */
    /** Unblock messages may still exist, so run a few more cycles.  */
    int unblock;
    unblock = 10;
    while (unblock--)
        tick_tock ();
}

void Simulator::add_repeater_event (Repeater_event *re)
{
    this->repeater_event_queue.insert(std::make_pair (re->execute_time, re));
}

void Simulator::process_repeater_event_queue()
{
    std::multimap<timestamp_t, Repeater_event *>::iterator front;

    /** Check for any repeater_events to process this cycle.
     *  if ready
     *      execute, replicate and remove original
     */
    if (!repeater_event_queue.empty())
    {
        while ((front = repeater_event_queue.begin())->second->execute_time <= Global_Clock)
        {
            //execute also re-assigned time for the next issue
            front->second->execute ();

            if (front->second->execute_time <= Global_Clock)
                fatal_error ("Possible infinite loop in the repeater queue!");

            this->add_repeater_event (front->second);

            /** OK because front is reassigned before any future reference.  */
            repeater_event_queue.erase (front);
        }
    }
}

void Simulator::collect_qset_stat ()
{
    int qset_1lvl_cnt = 0;

    for(int i = 0 ; i < settings.num_nodes; i++)
    {
        if(this->get_L1(i)->follower_sets_mode == QSET_1LVL)
            qset_1lvl_cnt++;
    }

    this->stats->qset_1lvl_sampled_timestamp.collect(Global_Clock);
    this->stats->qset_1lvl_sampled_count.collect(qset_1lvl_cnt);
}

/** Module return functions.  */
Mesh_interface* Simulator::get_NI (int node)
{
    return (Mesh_interface *)(Nd[node]->mod[NI_M]);
}
processor* Simulator::get_PR (int node)
{
    return (processor*)(Nd[node]->mod[PR_M]);
}
Hash_table* Simulator::get_L1 (int node)
{
    return (Hash_table *)(Nd[node]->mod[L1_M]);
}
Hash_table* Simulator::get_L2 (int node)
{
    return (Hash_table *)(Nd[node]->mod[L2_M]);
}
Hash_table* Simulator::get_L3 (int node)
{
    return (Hash_table *)(Nd[node]->mod[L3_M]);
}
Memory_controller* Simulator::get_MC (int node)
{
    return (Memory_controller *)(Nd[node]->mod[MC_M]);
}

/** Debug.  */
void Simulator::dump_processors (void)
{
    processor *pr;

    /*    for (int i = 0; i < settings.num_nodes; i++)
          {
          pr = get_PR (i);
          if (pr->active_contexts () == 0)
          {
          fprintf (stderr, "Core %4d: idle\n", i);
          }
          else 
          {
          fprintf (stderr, "Core %4d: running %d threads:\n", i, pr->active_contexts ());
    //TODO: pr->dump_threads ();
    pr->dump_contexts ();
    }
    }
     */
}

void Simulator::dump_outstanding_requests (int nodeID)
{
    assert (nodeID < settings.num_nodes);
    //	get_PR (nodeID)->dump_contexts ();
}

void Simulator::dump_cache_block (int nodeID, paddr_t addr)
{
    assert (get_L1(nodeID));
    get_L1 (nodeID)->dump_hash_entry (addr);
}



/** Simulator Stat Engine **/
Simulator_stat_engine::Simulator_stat_engine ()
    : Stat_engine (NULL),
    cycle_count ("Cycle Count",""),
    qset_1lvl_hist ("Qset 1lvl hist:", "", Sim.settings.num_nodes, 1, 0),
    qset_1lvl_sampled_timestamp("Qset 1lvl sampled time (x coor)", ""),
    qset_1lvl_sampled_count("Qset 1lvl sampled count (y coor)", ""),
    qset_1lvl_sampled_latency ("1lvl sampled latency", ""),
    qset_2lvl_sampled_latency ("2lvl sampled latency", ""),
    qset_1lvl_sampled_latency_cnt ("1lvl sampled latency cnt", ""),
    qset_2lvl_sampled_latency_cnt ("2lvl sampled latency cnt", ""),
    active_threads_hist ("Active threads hist:", "", Sim.settings.num_nodes*Sim.settings.threads_per_processor, 1, 0)
{
}

Simulator_stat_engine::~Simulator_stat_engine ()
{
}

void Simulator_stat_engine::global_stat_merge (Stat_engine * e)
{
    // don't forget to cast the engine to the appropriate type
    //Simulator_stat_engine * global_engine = (Simulator_stat_engine*)e;
}

void Simulator_stat_engine::print_stats (ostream & out)
{
    int i;
    double dir_requests;

    cycle_count.print (out);

    dir_requests = 0;
    for (i = 0; i < Sim.settings.num_nodes; i++)
    {
        dir_requests +=
            Sim.get_L2 (i)->stats->message_stats[C_GETIS] +
            Sim.get_L2 (i)->stats->message_stats[C_GETIM] +
            Sim.get_L2 (i)->stats->message_stats[C_GETSM];
    }
    out << "Throughput (L2)," << dir_requests/((double)Global_Clock) << endl;

    if (Sim.settings.dir_mode != DIR_1L)
    {
        dir_requests = 0;
        for (i = 0; i < Sim.settings.num_nodes; i++)
        {
            dir_requests +=
                Sim.get_L3 (i)->stats->message_stats[C_GETIS] +
                Sim.get_L3 (i)->stats->message_stats[C_GETIM] +
                Sim.get_L3 (i)->stats->message_stats[C_GETSM];
        }
        out << "Throughput (L3)," << dir_requests/((double)Global_Clock) << endl;
    }

    qset_1lvl_hist.print (out);
    qset_1lvl_sampled_timestamp.print (out);
    qset_1lvl_sampled_count.print (out);
    qset_1lvl_sampled_latency.print (out);
    qset_2lvl_sampled_latency.print (out);
    qset_1lvl_sampled_latency_cnt.print (out);
    qset_2lvl_sampled_latency_cnt.print (out);
    active_threads_hist.print (out);

}

void Simulator_stat_engine::clear_stats ()
{
    cycle_count.clear ();
    qset_1lvl_hist.clear ();
    qset_1lvl_sampled_timestamp.clear ();
    qset_1lvl_sampled_count.clear ();
    qset_1lvl_sampled_latency.clear ();
    qset_2lvl_sampled_latency.clear ();
    qset_1lvl_sampled_latency_cnt.clear ();
    qset_2lvl_sampled_latency_cnt.clear ();
    active_threads_hist.clear ();

}

void Simulator_stat_engine::start_warmup ()
{
}

void Simulator_stat_engine::end_warmup ()
{
}

void Simulator_stat_engine::save_samples ()
{
}
